# -*- coding: UTF-8 -*- 
#auther = xiang
# date 2017/06/26
#python 3.x

from lxml import etree
import lxml
import urllib.request
import urllib.error
from bs4 import BeautifulSoup

input = input("正在读取，Enter查看，Q退出")

class duanzi:
    def __init__(self):
        self.page_number = 1
        self.user_agent = 'Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)'
        self.headers = {'User-Agent' :self.user_agent}
        self.stories = {}
        self.enable = False

    def getpage(self,page_number):
        try:
            url = "https://www.pengfu.com/xiaohua_%s.html"%(page_number)
            request = urllib.request.Request(url,headers=self.headers)
            response = urllib.request.urlopen(request)
            data = response.read().decode('utf-8')
            return data
        except urllib.error as e:
            if hasattr(e,"reason"):
                print("error",e.reason)
                return None

    def getpage_item(self,page_number):
        page_storie=[]
        data = self.getpage(page_number)
        if not data :
            print("捧腹网段子加载失败,请查看网络")
            return None
        tree=etree.HTML(data)
        #标题
        titles = tree.xpath('//div/dl/dd/h1/a/text()')
        #内容
        soup = BeautifulSoup(data,"lxml")
        txts = soup.find_all('div',attrs={"class":"content-img clearfix pt10 relative"})
        for txt in txts :
            div_br = txt.text
            div_br=div_br.split()
            page_storie.append(div_br)
        temp_duanzi = dict(zip(titles, page_storie))
        print(temp_duanzi)
        return temp_duanzi

    def load_page(self):
        if self.enable==True:
            if len(self.stories)<2:
                temp_duanzi = self.getpage_item(self.page_number)
                if temp_duanzi:
                    self.stories.update(temp_duanzi.items())
                    self.page_number +=1


    def get_one(self,temp_duanzi,page_number):
        for story in temp_duanzi:
            self.load_page()
            if input == "Q":
                self.enable = False
                return
            print("第%s\t页,段子为%s\n"%(page_number,story[0]))
            self.stories.pop(story[0],"去重失败")

    def start(self):
        print("正在读取，Enter查看，Q退出")
        self.enable = True
        self.load_page()
        now_page = 0
        while self.enable:
            if len(self.stories)>0:
                temp_duanzi= self.stories
                now_page +=1
                self.get_one(temp_duanzi,now_page)

if __name__ == '__main__':
    spider = duanzi()
    spider.start()


